{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MountainCarContinuous-v0 with PPO, Vectorized Environment\n",
    "\n",
    "\n",
    "### 1. Create Vectorized Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gym version:  0.17.3\n",
      "torch version:  1.2.0\n",
      "device:  cpu\n",
      "max_steps:  999\n",
      "threshold:  90.0\n",
      "reassigned threshold:  150\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from  collections  import deque\n",
    "import time\n",
    "from model import Policy\n",
    "from ppo import ppo_agent\n",
    "from storage import RolloutStorage\n",
    "from utils import get_render_func, get_vec_normalize\n",
    "from envs import make_vec_envs\n",
    "from parallelEnv import parallelEnv\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('gym version: ', gym.__version__)\n",
    "print('torch version: ', torch.__version__)\n",
    "\n",
    "seed = 0 \n",
    "gamma=0.99\n",
    "num_processes =  16 \n",
    "\n",
    "device = torch.device(\"cpu\")\n",
    "print('device: ', device)\n",
    "\n",
    "envs = parallelEnv('MountainCarContinuous-v0', n=num_processes, seed=seed)\n",
    "\n",
    "## make_vec_envs -cannot find context for 'forkserver'\n",
    "## forkserver is only available in Python 3.4+ and only on some Unix platforms (not on Windows).\n",
    "## envs = make_vec_envs('BipedalWalker-v2', \\\n",
    "##                    seed + 1000, num_processes,\n",
    "##                    None, None, False, device='cpu', allow_early_resets=False)\n",
    "\n",
    "max_steps = envs.max_steps\n",
    "print('max_steps: ', max_steps)\n",
    "\n",
    "threshold = envs.threshold\n",
    "print('threshold: ', threshold)\n",
    "threshold = 150\n",
    "print('reassigned threshold: ', threshold)\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "dir_chk = 'dir_save_test'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Instantiate Model, Agent and Storage\n",
    "\n",
    "Initialize the Policy (model MLPBase), PPO Agent and Rollout Storage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type obs:  <class 'numpy.ndarray'> , shape obs:  (16, 2)\n",
      "type obs_t:  <class 'torch.Tensor'> , shape obs_t:  torch.Size([16, 2])\n"
     ]
    }
   ],
   "source": [
    "## model Policy uses MLPBase\n",
    "policy = Policy(envs.observation_space.shape, envs.action_space,\\\n",
    "        base_kwargs={'recurrent': False})\n",
    "\n",
    "policy.to(device)\n",
    "\n",
    "agent = ppo_agent(actor_critic=policy, ppo_epoch=16, num_mini_batch=16,\\\n",
    "                 lr=0.01, eps=1e-5, max_grad_norm=0.5)\n",
    "\n",
    "rollouts = RolloutStorage(num_steps=max_steps, num_processes=num_processes, \\\n",
    "                        obs_shape=envs.observation_space.shape, action_space=envs.action_space, \\\n",
    "                        recurrent_hidden_state_size=policy.recurrent_hidden_state_size)\n",
    "\n",
    "obs = envs.reset()\n",
    "print('type obs: ', type(obs), ', shape obs: ', obs.shape)\n",
    "obs_t = torch.tensor(obs)\n",
    "print('type obs_t: ', type(obs_t), ', shape obs_t: ', obs_t.shape)\n",
    "\n",
    "rollouts.obs[0].copy_(obs_t)\n",
    "rollouts.to(device)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.Save model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(model, directory, filename, suffix):\n",
    "    torch.save(model.base.actor.state_dict(), '%s/%s_actor_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(model.base.critic.state_dict(), '%s/%s_critic_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(model.base.critic_linear.state_dict(), '%s/%s_critic_linear_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(model.base, '%s/%s_model_base_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(model.dist, '%s/%s_model_dist_%s.pth' % (directory, filename, suffix))\n",
    "    \n",
    "limits = [-300, -160, -100, -70, -50, 0, 20, 30, 40, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330]\n",
    "\n",
    "def return_suffix(j):\n",
    "    suf = '0'\n",
    "    for i in range(len(limits)-1):\n",
    "        if j > limits[i] and j < limits[i+1]:\n",
    "            suf = str(limits[i+1])\n",
    "            break\n",
    "        \n",
    "        i_last = len(limits)-1    \n",
    "        if  j > limits[i_last]:\n",
    "            suf = str(limits[i_last])\n",
    "            break\n",
    "    return suf      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train the Agent  with Vectorized Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "num_updates=1000000\n",
    "gamma = 0.99\n",
    "tau=0.95\n",
    "save_interval=30\n",
    "log_interval= 1 \n",
    "\n",
    "def ppo_vec_env_train(envs, agent, policy, num_processes, num_steps, rollouts):\n",
    "    \n",
    "    time_start = time.time()\n",
    "    \n",
    "    n=len(envs.ps)    \n",
    "    envs.reset()\n",
    "    \n",
    "    # start all parallel agents\n",
    "    print('Number of agents: ', n)\n",
    "    envs.step([[1]*4]*n)\n",
    "    \n",
    "    indices = []\n",
    "    for i  in range(n):\n",
    "        indices.append(i)\n",
    "     \n",
    "    s = 0\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = []    \n",
    "\n",
    "    for i_episode in range(num_updates):\n",
    "        \n",
    "        total_reward = np.zeros(n)\n",
    "        timestep = 0\n",
    "        \n",
    "        done = False\n",
    "        \n",
    "        for timestep in range(num_steps):\n",
    "            \n",
    "            with torch.no_grad():\n",
    "                value, actions, action_log_prob, recurrent_hidden_states = \\\n",
    "                   policy.act(\n",
    "                        rollouts.obs[timestep],\n",
    "                        rollouts.recurrent_hidden_states[timestep],\n",
    "                        rollouts.masks[timestep])\n",
    "                   \n",
    "                \n",
    "            obs, rewards, done, _ = envs.step(actions.cpu().detach().numpy())\n",
    "            \n",
    "            \n",
    "            total_reward += rewards  ## this is the list by agents\n",
    "                        \n",
    "            # If done then clean the history of observations.\n",
    "            masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done])\n",
    "            obs_t = torch.tensor(obs)\n",
    "            \n",
    "            ## Add one dimnesion to tensor, \n",
    "            ## This is (unsqueeze(1)) solution for:\n",
    "            ## RuntimeError: The expanded size of the tensor (1) must match the existing size...\n",
    "            rewards_t = torch.tensor(rewards).unsqueeze(1)\n",
    "            rollouts.insert(obs_t, recurrent_hidden_states, actions, action_log_prob, \\\n",
    "                value, rewards_t, masks)\n",
    "                                \n",
    "        avg_total_reward = np.mean(total_reward)\n",
    "        scores_deque.append(avg_total_reward)\n",
    "        scores_array.append(avg_total_reward)\n",
    "                \n",
    "        with torch.no_grad():\n",
    "            next_value = policy.get_value(rollouts.obs[-1],\n",
    "                            rollouts.recurrent_hidden_states[-1],\n",
    "                            rollouts.masks[-1]).detach()\n",
    "\n",
    "        rollouts.compute_returns(next_value, gamma, tau)\n",
    "\n",
    "        agent.update(rollouts)\n",
    "\n",
    "        rollouts.after_update()\n",
    "        \n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "\n",
    "        if i_episode > 0 and i_episode % save_interval == 0:\n",
    "            print('Saving model, i_episode: ', i_episode, '\\n')\n",
    "            suf = return_suffix(avg_score)\n",
    "            save(policy, dir_chk, 'we0', suf)\n",
    "\n",
    "        \n",
    "        if i_episode % log_interval == 0 and len(scores_deque) > 1:            \n",
    "            prev_s = s\n",
    "            s = (int)(time.time() - time_start)\n",
    "            t_del = s - prev_s\n",
    "            print('Ep. {}, Timesteps {}, Score.Agents: {:.2f}, Avg.Score: {:.2f}, Time: {:02}:{:02}:{:02}, \\\n",
    "Interval: {:02}:{:02}'\\\n",
    "                   .format(i_episode, timestep+1, \\\n",
    "                        avg_total_reward, avg_score, s//3600, s%3600//60, s%60, t_del%3600//60, t_del%60)) \n",
    "    \n",
    "        if len(scores_deque) > 1 and avg_score > threshold:   \n",
    "            print('Environment solved with Average Score: ',  avg_score )\n",
    "            break\n",
    "    \n",
    "    \n",
    "    return scores_array, avg_scores_array\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents:  16\n",
      "Ep. 1, Timesteps 999, Score.Agents: -100.28, Avg.Score: -104.26, Time: 00:00:07, Interval: 00:07\n",
      "Ep. 2, Timesteps 999, Score.Agents: -89.06, Avg.Score: -99.19, Time: 00:00:11, Interval: 00:04\n",
      "Ep. 3, Timesteps 999, Score.Agents: -81.92, Avg.Score: -94.87, Time: 00:00:15, Interval: 00:04\n",
      "Ep. 4, Timesteps 999, Score.Agents: -72.26, Avg.Score: -90.35, Time: 00:00:19, Interval: 00:04\n",
      "Ep. 5, Timesteps 999, Score.Agents: -64.13, Avg.Score: -85.98, Time: 00:00:23, Interval: 00:04\n",
      "Ep. 6, Timesteps 999, Score.Agents: -56.60, Avg.Score: -81.78, Time: 00:00:27, Interval: 00:04\n",
      "Ep. 7, Timesteps 999, Score.Agents: -45.15, Avg.Score: -77.21, Time: 00:00:31, Interval: 00:04\n",
      "Ep. 8, Timesteps 999, Score.Agents: -15.89, Avg.Score: -70.39, Time: 00:00:35, Interval: 00:04\n",
      "Ep. 9, Timesteps 999, Score.Agents: 35.67, Avg.Score: -59.79, Time: 00:00:39, Interval: 00:04\n",
      "Ep. 10, Timesteps 999, Score.Agents: 78.21, Avg.Score: -47.24, Time: 00:00:42, Interval: 00:03\n",
      "Ep. 11, Timesteps 999, Score.Agents: 130.52, Avg.Score: -32.43, Time: 00:00:46, Interval: 00:04\n",
      "Ep. 12, Timesteps 999, Score.Agents: 191.98, Avg.Score: -15.17, Time: 00:00:50, Interval: 00:04\n",
      "Ep. 13, Timesteps 999, Score.Agents: 223.33, Avg.Score: 1.87, Time: 00:00:54, Interval: 00:04\n",
      "Ep. 14, Timesteps 999, Score.Agents: 284.03, Avg.Score: 20.68, Time: 00:00:58, Interval: 00:04\n",
      "Ep. 15, Timesteps 999, Score.Agents: 331.33, Avg.Score: 40.10, Time: 00:01:02, Interval: 00:04\n",
      "Ep. 16, Timesteps 999, Score.Agents: 394.99, Avg.Score: 60.97, Time: 00:01:06, Interval: 00:04\n",
      "Ep. 17, Timesteps 999, Score.Agents: 479.05, Avg.Score: 84.20, Time: 00:01:10, Interval: 00:04\n",
      "Ep. 18, Timesteps 999, Score.Agents: 518.25, Avg.Score: 107.04, Time: 00:01:14, Interval: 00:04\n",
      "Ep. 19, Timesteps 999, Score.Agents: 553.75, Avg.Score: 129.38, Time: 00:01:18, Interval: 00:04\n",
      "Ep. 20, Timesteps 999, Score.Agents: 619.99, Avg.Score: 152.74, Time: 00:01:22, Interval: 00:04\n",
      "Environment solved with Average Score:  152.7411231610043\n"
     ]
    }
   ],
   "source": [
    "scores, avg_scores = ppo_vec_env_train(envs, agent, policy, num_processes, max_steps, rollouts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(model=policy,directory=dir_chk,filename='we0',suffix='final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  21 , len of avg_scores:  21\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAEGCAYAAACtn3UnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deVyVZf7/8dcFqIAoi+IGIm6oKK64VZaVLZqTZdliU2q21zQ10zbTfFtm+s000zTNTE2lbS5ZWbbYtK9mTZnivisuCC6Iiiyyw/X7474pRDBM4ObA+/l4nMe5z3Vf55wPN3D4cK3GWouIiIiIr/DzOgARERGRE6HkRURERHyKkhcRERHxKUpeRERExKcoeRERERGfEuB1AHWpbdu2NjY21uswRER8yvLlyw9YayO9jkOkOo06eYmNjSUpKcnrMEREfIoxJsXrGESOR91GIiIi4lOUvIiIiIhPUfIiIiIiPqVRj3kREZGGY/ny5e0CAgKeB/qhf56lemXAupKSkuuGDBmyv6oKSl5ERKReBAQEPN+hQ4c+kZGRmX5+ftpYT6pUVlZmMjIy4vft2/c8cGFVdZT5iohIfekXGRmZrcRFjsfPz89GRkZm4bTQVV2nHuMREZGmzU+Ji9SE+3NSbY6i5EVEpJH5cO1eFq7a7XUYInVGyYuISCOyOvUwd8xfxdzvUigtUyNHVe69994OPXr06BsXFxffu3fv+C+++KKl1zHJidGAXRGRRmL34Xyum5NEZKsWPHv1EPz9jNchNTifffZZy48//jhs7dq1G4KCguzevXsDCgsLf/aFKi4uplmzZrUZotSAWl5ERBqB3MISps9aRkFRKS9NHUrbkBZeh9Qg7d69u1lERERJUFCQBejYsWNJbGxs8VdffRU8aNCg3r169YpPSEjok5mZ6ZeXl2cuvfTS2Li4uPg+ffrE//e//20F8O9//7vN2LFju5111lk9Ro0aFQfwf//3f+379evXJy4uLv7OO+/s5OXX2BR42vJijAkDyuf8W+BaYDMwH4gFdgKXWWszjTEG+BcwDsgDplprV3gQtohIg1JaZrn91ZVs3Z/Li1OH0rN9K69D+kl3L1jdecu+nODafM24Dq3yHrt0QOrx6lx00UXZf/nLXzrFxsb2O+2007KvvPLKQ2efffaRq666qvu8efO2nXHGGXmHDh3yCwkJKXvkkUfaA2zZsmXDypUrA8eNG9dz27Zt6wBWrFgRsmbNmvXt27cvfeutt1onJycHrlmzZqO1ljFjxvT48MMPQ8aOHZtbm1+f/Mjrlpd/AR9Za3sDA4CNwH3A59bansDn7mOAsUBP93YD8Ez9hysi0vD8+YONfLFpPw/9Ip4z4rQZ9PGEhoaWrVu3bsNTTz2VEhkZWTJlypTujz/+eGS7du2KzzjjjDyAiIiIsmbNmvHtt9+GXHPNNQcBBg0aVNCpU6eitWvXBgKMGjUqu3379qUAH330UevFixe3jo+Pj+/bt2/8tm3bAjdt2hTo3VfZ+HnW8mKMaQ2cDkwFsNYWAUXGmAnAaLfabGARcC8wAZhjrbXAEmNMmDGmo7V2bz2HLiLSYMz7PoUXvtnB1FNiuXpkrNfh1NhPtZDUpYCAAMaPH58zfvz4nP79++c/++yzkcaYY0Y3O39uqhYcHFxWsd4dd9yx9+677z5QRyFLJV62vHQDMoCXjDErjTHPG2NaAu3LExL3vp1bPwqo+MOe5pYdxRhzgzEmyRiTlJGRUbdfgYiIh77emsEDC9dzZq9I/m98vNfh+ITVq1e3WLt27Q8DglauXBnUs2fPgvT09OZfffVVMEBmZqZfcXExp512Wu7LL78cAbBmzZoWe/fubd6/f/+Cyq85duzY7Llz57bNysryA9ixY0ez3bt3a0JMHfLy4gYAg4FfWWu/N8b8ix+7iKpS1WjwqjLlmcBMgMTERM0TFJFGKXl/DrfMW0HPdiE8OXmwZhbVUHZ2tv/tt98ek52d7e/v729jY2MLZ8+enbJly5YDt99+e0xBQYFfYGBg2eLFi7fcc889+6+++uoucXFx8f7+/syYMWNn+UDfiiZOnJi9fv36wKFDh/YGp1Vm3rx5O6Kiokrq/ytsGszxmsXq9I2N6QAssdbGuo9H4SQvPYDR1tq9xpiOwCJrbS9jzAz3+FW3/ubyetW9R2Jiok1KSqrrL0VEpF4dOlLERf/5H3lFpbxz6ylEh9fquFeMMcuttYm1+qLA6tWrdw4YMEBdK1Ijq1evbjtgwIDYqs551m1krd0HpBpjerlFZwMbgHeBKW7ZFGChe/wucI1xjACyNN5FRJqawpJSbpybRHp2Ac9dM6TWExcRX+B1n9yvgHnGmObAdmAaTkL1ujFmOrALmOTW/QBnmnQyzlTpafUfroiId6y13PfmWpbtzOSpyYMYFBPudUginvA0ebHWrgKqapo8u4q6Fri1zoMSEWmg/vNlMm+v3M1vzoljfH+tgyZNl9frvIiISA28t2YPf/9kCxcPiuJXZ/XwOhwRTyl5ERFp4FbuyuS3r68msUs4j16SgLPguEjTpeRFRKQBS8vM4/o5y2nXugUzrh5CiwB/r0MS8ZySFxGRBiq3sITrZidRWFzKi1OG0kabLdaKOXPmhBljhqxcubJBLOG/b98+/+HDh8cFBwcPuuaaa2Iqnvv666+D4+Li4mNiYvpNnTq1c1mZs7Bvenq6/ymnnNKzS5cu/U455ZSeGRkZtZLVnnHGGT0OHDhwUq/13nvvtTrzzDPrtG9TyYuISANUcbPFp3852Cc2W/QVr732WsTgwYNz586dG+F1LADBwcH2j3/8456HHnoorfK5W265pcvTTz+dsnPnznXbt28PXLBgQWuABx98sOPo0aNzUlJS1o0ePTrngQce6FAbsXz11VfJbdu2La2N16pLSl5ERBqY4tIyHli4ji827efhC/syqqc2W6wtWVlZfklJSSEvvfTSzrfffvuHueYXXHBBt/nz54eWP77kkktiZ82aFZaTk+M3bty4bnFxcfEXXHBBt/79+/devHjxMYvrLFy4sFWfPn3i4+Li4idNmhSbn59vAKKiohLuvPPOTvHx8X3i4uLiq2rtad26ddl5552XGxgYWFaxPCUlpVlubq7fmDFjjvj5+XHVVVcdfOedd8IBPvroo7Abb7zxIMCNN9548MMPPzxm3nxJSQk33nhjdL9+/frExcXFP/bYY23BaRlJTEzsdc4553Tv3r1738mTJ8eUljr5SlRUVMLevXsDsrOz/UaPHt2jV69e8T179uz73HPPhR/v61ywYEHrrl279h0yZEivBQsWhJXHkJ2d7Tdp0qTYfv369enTp0/8yy+/HAaQlJQUmJCQ0Kd3797xcXFx8RW3bKgJr9d5ERGRClbuyuR3b61l074crh/VlV+O6OJ1SHXjnVs7s39D7a6w1y4+j4v+c9wNH+fNmxc2evTorP79+xeGhYWVfvPNN8GnnXZa3uWXX35o/vz54ZdffnlWQUGB+d///td69uzZKX/729/ahYWFlW7ZsmXDsmXLAkeOHNm38mvm5eWZG2+8sesnn3yyuX///oUXX3xx7GOPPRb5wAMP7Ado27ZtyYYNGzY++uijkY8++mj7+fPnp9Tky0lJSWnWsWPH4vLHXbp0Kdq7d28zgIMHDwZ06dKl2C0vPnTo0DF/z//5z3+2DQ0NLV23bt3G/Px8M3To0N6/+MUvsgHWrl3bcuXKlevi4uKKTj/99J5z5swJnzZtWmb5c996663WHTp0KF60aFGy+37+1X2dd911V8Ztt90W++mnn27u27dv4fjx47uVv87vf//7jmeeeWb2G2+8sfPAgQP+iYmJfS688MLsJ598MvKWW25Jv/nmmw8VFBSYkpIT20lBLS8iIg1AbmEJD727nonPfMvhvGJmXj2E+y/QZou17fXXX4+48sorMwEuueSSQ+VdR5deemnWt99+2zo/P98sWLAgdNiwYTkhISH222+/DbnyyisPAQwdOrQgLi4ur/Jrrl69OjA6Orqwf//+hQBTp049+M033/zQzzd58uRMgGHDhuWlpqbWuIWhqu17TmSm2Weffdb69ddfb9O7d+/4QYMG9cnMzAzYsGFDIEBCQsKR+Pj4ooCAAC677LJDX3/9dUjF5w4ePDj/66+/bn3zzTdHffTRRyFt2rQpre7rXLVqVWB0dHRhQkJCYXkLUfnrLFq0qPUTTzzRsXfv3vGnnXZar8LCQpOcnNx85MiRRx5//PGO999/f4etW7c2DwkJOaG9itTyIiLisU83pPPAwnXsyy7gmhFduOu8XrQKbOZ1WHXrJ1pI6sK+ffv8lyxZ0nrLli1Bt912G6WlpcYYY5955pm04OBgO2LEiJy33nqr9fz588PLE5aa7P/3U3UCAwMtQEBAgC0pKalx9hEbG1tc3tICkJKS0rxDhw7FAG3atClJSUlp1qVLl+KUlJRmERERxzRdWGvN448/vuuSSy7Jrlj+3nvvtaqcBFV+3L9//8IVK1ZsePPNN0Pvv//+qM8++yx74sSJh6uLtbqkylrLggULkgcMGFBYsXzw4MEFo0aNOvL222+Hjh07Nu7pp5/eeeGFF+ZUezEqUcuLiIhH0rMLuPnl5Vw/J4nWgc148+ZTeHhCv8afuHhk7ty54RMnTjy4Z8+etbt37167b9++NdHR0UWffPJJCMAVV1xxaNasWW2XLVvWauLEidkAp5xySu5rr70WDrB8+fLALVu2BFV+3YEDBxbs3r27+bp161oAzJkzp82oUaNq/Ie4Ol26dClu2bJl2eeff96yrKyMefPmtZkwYcJhgPPOO+/wjBkz2gDMmDGjzfnnn39MYnHOOedkPfPMM5GFhYUGYM2aNS2ys7P9wOk22rRpU/PS0lIWLFgQUTnenTt3NmvVqlXZLbfccuiOO+5IX7VqVXB1X+fAgQML0tLSmq9fv74FOAOiy1/nzDPPzH788cfbl8+S+t///hcEsGHDhuZ9+vQp/MMf/rD/3HPPPbxq1apjruvxqOVFRKSelZVZXlm6i79+uInC0jLuPq8XN5zejWb++n+yLr3xxhtt7rnnnqM29J0wYULm3LlzI84///zciy++OPumm27qOmbMmMPlrSV33313xmWXXRYbFxcX369fv7xevXrlh4eHHzUbJzg42D777LM7J02a1L20tJQBAwbk3XXXXRknEltUVFRCbm6uf3Fxsfn444/DPvjggy1DhgwpePrpp1OmT5/etaCgwJx55pnZkyZNygJ4+OGH91588cXdu3Tp0rZTp05F77zzzrbKr3nnnXce2LlzZ4uEhIQ+1loTERFR/MEHH2wDGDhwYO5vf/vb6E2bNgUNHz485+qrrz4q+Vm+fHnQ7373u2g/Pz8CAgLs008/nVLd1xkUFGSffPLJlPHjx/eIiIgoGT58eO7GjRuDAB599NE9N9xwQ0zv3r3jrbUmOjq68Msvv0yeO3duxBtvvNEmICDARkZGFv/lL3/ZcyLXy9SkScxXJSYm2qSkJK/DEBH5wdb0HH731lqSUjI5pXsb/t/FCXRt29LrsI5ijFlura1q37mTsnr16p0DBgw4UNuvW5dKSkooKioywcHBdv369S3OPffcuG3btq0rT2580Xvvvdfq8ccfb//ll18mex3L8axevbrtgAEDYqs6p5YXEZF6UFBcytOLtvHMomRatgjg75MGcMngKC3138Dl5OT4jRo1qldxcbGx1vLEE0+k+HLi0lgoeRERqWNLth/k92+vZXvGES4a2Ik/jI+nrVbL9Qnh4eFl69at2+h1HLVp/PjxOePHjz/pMTleUvIiIlJHsvKK+cuHG3ltWSqdI4KYfe0wzohr0gvOlZWVlRk/Pz+1XMhxlZWVGaCsuvNKXkRE6sDnG9O59821ZOYVcePp3fj1mJ4EN2/yH7nrMjIy4iMjI7OUwEh1ysrKTEZGRiiwrro6Tf43SUSkNuUVlfCn9zby6tJd9O7QilnThtIvKvSnn9gElJSUXLdv377n9+3b1w8t1SHVKwPWlZSUXFddBSUvIiK1ZOWuTO6cv4qUQ3nceHo3fnNuHC0CamWz30ZhyJAh+4ELvY5DfJ+nyYsxZieQA5QCJdbaRGNMBDAfiAV2ApdZazONMyT/X8A4IA+Yaq1d4UXcIiIVlZSW8eQXyTz1ZTIdWgfyynUjGNm9jddhiTRaDaHl5UxrbcV5//cBn1trHzXG3Oc+vhcYC/R0b8OBZ9x7ERHP7DhwhDvmr2J16mEuHhTFwxP60lor5IrUqYaQvFQ2ARjtHs8GFuEkLxOAOdZZVW+JMSbMGNPRWru3ylcREalD1lpeXZrKn97bQPMAP56aPIjx/Tt5HZZIk+B18mKBT4wxFphhrZ0JtC9PSKy1e40x7dy6UUDFjbzS3LKjkhdjzA3ADQAxMTF1HL6INEUZOYXc9+YaPt+0n9N6tOXvkwbQITTQ67BEmgyvk5dTrbV73ATlU2PMpuPUrWoZymOm2rkJ0ExwtgeonTBFRByfbUjn3jfXkFNYwgPj45l6Six+flolV6Q+eZq8WGv3uPf7jTFvA8OA9PLuIGNMR2C/Wz0N6Fzh6dHACW3kJCLycx0pLOGR9zfw6tJU4ju25tUrBhLXvpXXYYk0SZ7NszfGtDTGtCo/Bs7FWZDmXWCKW20KsNA9fhe4xjhGAFka7yIi9WHFrkwu+PfXvLYslZvO6M7bt56ixEXEQ162vLQH3nY3JQsAXrHWfmSMWQa8boyZDuwCJrn1P8CZJp2MM1V6Wv2HLCJNSUlpGf/+Ipn/uFOgX7t+BMO7aQq0iNc8S16stduBAVWUHwTOrqLcArfWQ2giIgD849MtPL1oGxMHR/HQhZoCLdJQeD1gV0SkQVqdephnv9rGZYnR/O3SY/7PEhEPaW8JEZFKCopLueuN1bRvHcgfxsd7HY6IVKKWFxGRSv71+Va27s9l9rXD1FUk0gCp5UVEpIJVqYeZ8dU2Lk/szBlxkV6HIyJVUPIiIuIqKC7lbre76P7xfbwOR0SqoW4jERGXuotEfINaXkREUHeRiC9R8iIiTV7F2UXqLhJp+NRtJCJN3j8/20qyuotEfIZaXkSkSVuVepiZi9VdJOJLlLyISJOl7iIR36RuIxFpstRdJOKb1PIiIk3Syl2ZzFy8jSuGqrtIxNcoeRGRJqeguJS7F6yhfetAfn+BuotEfI26jUSkyVF3kYhvU8uLiDQp6i4S8X1KXkSkySifXdShdSD3q7tIxGep20hEmox/fraVbRlHmH3tMFqpu0jEZ3ne8mKM8TfGrDTGvOc+7mqM+d4Ys9UYM98Y09wtb+E+TnbPx3oZt4j4FnUXiTQenicvwK+BjRUe/xV4wlrbE8gEprvl04FMa20P4Am3nojIT1J3kUjj4mnyYoyJBi4AnncfG+AsYIFbZTZwkXs8wX2Me/5st76IyHE98dkWtmUc4dFL+qu7SKQR8Lrl5Z/APUCZ+7gNcNhaW+I+TgOi3OMoIBXAPZ/l1j+KMeYGY0ySMSYpIyOjLmMXER+wclcmzy3ezhVDO3O6uotEGgXPkhdjzHhgv7V2ecXiKqraGpz7scDamdbaRGttYmSkPqhEmrI9h/O56eXldAwNUneRSCPi5WyjU4ELjTHjgECgNU5LTJgxJsBtXYkG9rj104DOQJoxJgAIBQ7Vf9gi4guy8ouZ+tJS8gpLeeNmzS4SaUw8a3mx1v7OWhttrY0FrgC+sNZeBXwJXOpWmwIsdI/fdR/jnv/CWntMy4uISGFJKTfOTWLHgSPMuHoIvTu09jokEalFXo95qcq9wG+MMck4Y1pecMtfANq45b8B7vMoPhFpwMrKLPcsWMOS7Yf426X9OaVHW69DEpFa1iAWqbPWLgIWucfbgWFV1CkAJtVrYCLicx77ZDMLV+3h7vN6cfGgaK/DEZE60BBbXkREfpa5S1J4ZtE2Jg+P4ZbR3b0OR0TqiJIXEWkUPt2QzoML13F273b88cK+aBkokcZLyYuI+LyVuzL51asr6BcVypOTBxHgr482kcZMv+Ei4tNSDh7hutlJRLZqwQtThhLcvEEM5ROROqTkRUR81sHcQqa8uJQya5k9bRiRrVp4HZKI1AP9iyIiPim/qJTr5iSxN6uAV64fTrfIEK9DEpF6ouRFRHxOaZnl16+tZFXqYZ65ajBDukR4HZKI1CN1G4mIT7HW8sf/rueTDek8MD6e8/t19DokEalnSl5ExKc89/V2Zn+XwvWjujLt1K5ehyMiHlDyIiI+493Ve/jzB5u4oH9HfjdWu0SLNFVKXkTEJyzZfpC7Xl/NsNgIHp80AD8/LUIn0lQpeRGRBm9reg43zEmic0QQM68ZQmAzf69DEhEPKXkRkQZtVephJj//PS2a+TNr2jDCgpt7HZKIeEzJi4g0WO+t2cPlM76jRYAfr1w3nM4RwV6HJCINgNZ5EZEGx1rLk18k849PtzCkSzgzrx5CmxCtnisiDiUvItKgFBSXct+ba3hn1R4uHhTFXyYmaIyLiBxFyYuINBgHcgu5ce5ylqdkcte5cdx6Zg+M0awiETmakhcRaRA278th+uxlZOQU8p/Jg7mgv1bOFZGqeTZg1xgTaIxZaoxZbYxZb4x52C3vaoz53hiz1Rgz3xjT3C1v4T5Ods/HehW7iNSuRZv3c8kz31JYUsbrN45U4iIix+XlbKNC4Cxr7QBgIHC+MWYE8FfgCWttTyATmO7Wnw5kWmt7AE+49UTEx8363w6unbWMzhHBLLz1VAZ0DvM6JBFp4DxLXqwj133YzL1Z4CxggVs+G7jIPZ7gPsY9f7ZRZ7iIzyopLeOBhet46L8bOKt3exbcNJJOYUFehyUiPsDTdV6MMf7GmFXAfuBTYBtw2Fpb4lZJA6Lc4yggFcA9nwW0qeI1bzDGJBljkjIyMur6SxCRnyG7oJhps5Yx57sUbji9GzOuHkLLFhqCJyI1U+PkxRhzmjFmmnscaYw56e1crbWl1tqBQDQwDKhqpzVbHsJxzlV8zZnW2kRrbWJkZOTJhigitWzXwTwmPv0t3207yF8vSeD34/rgr32KROQE1OhfHWPMg0Ai0At4CaeL52Xg1NoIwlp72BizCBgBhBljAtzWlWhgj1stDegMpBljAoBQ4FBtvL+I1I9lOw9x49zllJZZ5kwfxind23odkoj4oJq2vFwMXAgcAbDW7gFancwbu603Ye5xEDAG2Ah8CVzqVpsCLHSP33Uf457/wlp7TMuLiDRMb61I46rnvic0qBlv33KKEhcR+dlq2slcZK21xhgLYIxpWQvv3RGYbYzxx0miXrfWvmeM2QC8Zox5BFgJvODWfwGYa4xJxmlxuaIWYhCRevDm8jR++8ZqRnSL4NlfDtHmiiJyUmqavLxujJmB06VzPXAt8NzJvLG1dg0wqIry7TjjXyqXFwCTTuY9RaT+fbohnXveXMOpPdrw4tShtAjQUv8icnJqlLxYa/9ujDkHyMYZ9/KAtfbTOo1MRHzeku0HufWVFfTr1JoZVycqcRGRWvGTyYvbrfOxtXYMznRmEZGftG53FtfPTiImIpiXpg0jRFOhRaSW/OSAXWttKZBnjAmth3hEpBHYnpHLlBeX0jqoGXOnDyOipca4iEjtqem/QgXAWmPMp7gzjgCstbfXSVQi4rP2ZuVz9QtLscCc6cPoGKpVc0WkdtU0eXnfvYmIVCvzSBHXvLCUrPxiXr1+BN0jQ7wOSUQaoZoO2J3t7u4c5xZtttYW111YIuJrjhSWMG3WMlIO5TF72jASotXTLCJ1o6Yr7I7G2RRxJ84y/Z2NMVOstYvrLjQR8RWFJaXc9PJy1qQd5plfDmFk92O2HRMRqTU17TZ6HDjXWrsZwBgTB7wKDKmrwETEN5SWWX4zfzVfbz3A3y7tz3l9O3gdkog0cjXdHqBZeeICYK3dgrO/kYg0YdZa/vDOOt5fu5f7x/XhssTOXockIk1ATVtekowxLwBz3cdXAcvrJiQR8RV//2Qzry7dxS2ju3P96d28DkdEmoiaJi83A7cCt+OMeVkMPF1XQYlIw/f819v5z5fbuHJYDHef18vrcESkCalp8hIA/Mta+w/4YdXdFnUWlYg0aAuWp/HI+xsZl9CBRy7qhzHG65BEpAmp6ZiXz4GKK00FAZ/Vfjgi0tB9sn4f9765hlE92/LE5QPx91PiIiL1q6bJS6C1Nrf8gXscXDchiUhD9d22g9z26kr6RYXy7C+HaKNFEfFETZOXI8aYweUPjDGJQH7dhCQiDU1pmWX+sl1cPyeJLhHBzJo6lJbaaFFEPFLTT587gDeMMXsAC3QCLq+zqESkwfh6awb/7/2NbNqXw5Au4Tw1eRDh2mhRRDx03OTFGDMUSLXWLjPG9AZuBCYCHwE76iE+EfHIlvQc/vzBRhZtzqBzRBD/mTyYcQkdNDhXRDz3Uy0vM4Ax7vFI4PfAr4CBwEzg0roLTUS8kJFTyBOfbeG1pbto2SKA+8f14ZpTumh8i4g0GD+VvPhbaw+5x5cDM621bwJvGmNWncwbG2M6A3OADkCZ+9r/MsZEAPOBWJy9lC6z1mYa59+9fwHjgDxgqrV2xcnEICI/Kigu5YVvdvDMom0UFJdyzchYfn12T3URiUiD85PJizEmwFpbApwN3HACz/0pJcBvrbUrjDGtgOXGmE+BqcDn1tpHjTH3AfcB9wJjgZ7ubTjwjHsvIiehrMyycPVuHvtoM3uyCjg3vj33je1Nt8gQr0MTEanSTyUgrwJfGWMO4Mwu+hrAGNMDyDqZN7bW7gX2usc5xpiNQBQwARjtVpsNLMJJXiYAc6y1FlhijAkzxnR0X0dEfoalOw7xyPsbWJOWRb+o1vzj8oGM6KYdoUWkYTtu8mKt/X/GmM+BjsAnbuIAzhTrX9VWEMaYWGAQ8D3QvjwhsdbuNca0c6tFAakVnpbmlh2VvBhjbsBtIYqJiamtEEUalR0HjvDohxv5eH06HUMD+cdlA7hoYBR+WnBORHzAT3b9WGuXVFG2pbYCMMaEAG8Cd1hrs48zk6GqE/aYAmtn4gwmJjEx8ZjzIk1ZTkEx//h0C3O/S6FFgB93nRvH9NO6EcEDUfkAAB8NSURBVNRcg3FFxHd4usqUMaYZTuIyz1r7llucXt4dZIzpCOx3y9OAzhWeHg3sqb9oRXxbflEpU19axspdmVw+tDN3nhNHu1aBXoclInLCarrCbq1zZw+9AGws3/DR9S4wxT2eAiysUH6NcYwAsjTeRaRmikvLuPWVFazYlclTkwfzl4n9lbiIiM/ysuXlVOBqYG2Fade/Bx4FXjfGTAd2AZPccx/gTJNOxpkqPa1+wxXxTdZa7ntzLV9s2s8jF/VjXEJHr0MSETkpniUv1tpvqHocCzjTsivXt8CtdRqUSCP06IebeHNFGneOieOXI7p4HY6IyEnzrNtIROrezMXbmLF4O9eM7MLtZ/fwOhwRkVqh5EWkkVqwPI0/f7CJC/p35MFf9NWeRCLSaCh5EWmEPt+Yzr1vruHUHm34x2UD8Nf6LSLSiCh5EWlkknYe4tZXVhDfsTUzrk7Uhooi0ugoeRFpRDbvy+HaWcvoGBrErGlDCWnh6VJOIiJ1QsmLSCORlpnHNS9+T2Azf+ZcO4w2IS28DklEpE7o3zKRRuBgbiHXvLCU/KJSXr9pJJ0jgr0OSbxyOBWWvwS2DMY85HU0InVCyYuIj8stLGHarGXsPpzPy9cNp3eH1l6HJPWtrAx2LIKlz8OWD52yvhPBWtAsM2mElLyI+LCikjJumruc9XuymfHLIQyNjfA6JKlP+Ydh1SuQ9AIcTIbgtnDqHZA4DcJivI5OpM4oeRHxUWVllt+8vopvkg/w2KX9GRPf3uuQpL7sXQPLnoe1b0BxHkQPg4nPQfwECNBYJ2n8lLyI+CBrLQ//dz3vrdnLfWN7Mymx808/SXxbSSFseBeWPQep30NAEPSfBEOvg44DvI5OpF4peRHxQU99kczs71K47rSu3Hh6N6/DkbpUPgB3xRw4kgER3eG8v8DAKyEo3OvoRDyh5EXEx8z9biePf7qFiYOi+P24Plr2vzGqagBu3PlOK0u3M8FPq1xI06bkRcSHPPvVNh79cBNn927HXy/tj5+W/W9cctJh1cuwfDYcTtEAXJFqKHkR8QHWWv728WaeWbSN8f078o/LBtLMX/99NwplZbD9C1g+CzZ/CGUlEDsKzn4A+vxCA3BFqqDkRaSBKy2zPLBwHfO+38Xk4TH8aUI/bbTYGGTvdVpZVsyBw7sguA2MuAUGT4G2PbyOTqRBU/Ii0oAVlZTx2zdW89/Ve7h5dHfuOa+Xxrj4srJS2FahlcWWQtfTYczD0PsCtbKI1JCSF5EGKr+olJvnLWfR5gzuPb83N4/u7nVI8nNl74GVbitLViq0jIRTfgWDr4E2+r6KnChPkxdjzIvAeGC/tbafWxYBzAdigZ3AZdbaTOP8u/kvYByQB0y11q7wIm6RupZdUMx1s5JYlnKIP1+cwOThGqzpc8pKIfkzp5Vly0fOXkPdRsO5j0CvcRDQ3OMARXyX1y0vs4CngDkVyu4DPrfWPmqMuc99fC8wFujp3oYDz7j3Io3KgdxCpry4lM37cvj3FYP4xYBOXockJyInHVbOcWYMZaVCy3Zw6q+dVpYIrckjUhs8TV6stYuNMbGViicAo93j2cAinORlAjDHWmuBJcaYMGNMR2vt3vqJVqTu7T6cz9XPf8+erHyem5LImb3aeR2S1IS1sGOxs8fQpvedGUNdz3BaWXpfAP7NvI5QpFHxuuWlKu3LExJr7V5jTPmndxSQWqFemlt2VPJijLkBuAEgJkZN7eI7tmXkcvXz35NTUMLc6cO1yaIvyDvkbIy4/CVnY8TAMBh+EwyZphlDInWoISYv1alqioU9psDamcBMgMTExGPOizRE63ZnMeXFpQC8esMI+kWFehyRVMtaSFsGSS/CuregtBA6D4fT73Y2RmwW5HWEIo1eQ0xe0su7g4wxHYH9bnkaUHH3uWhgT71HJ1LLlu44xPRZy2gVGMDL1w2nW2SI1yFJVQpzYM3rkPQSpK+F5iEw6JfO6rcdEryOTqRJaYjJy7vAFOBR935hhfLbjDGv4QzUzdJ4F/F1X27az83zltMpLIiXpw+nU5j+a29w9q11WlnWvA5FudA+AcY/AQmToEUrr6MTaZK8nir9Ks7g3LbGmDTgQZyk5XVjzHRgFzDJrf4BzjTpZJyp0tPqPWCRWvTf1Xu4c/4qenVoxexrh9E2RAuUNRjFBbBhISx7HtKWQkAg9J0IiddCdCJooUART3k92+jKak6dXUVdC9xatxGJ1I+5S1J4YOE6hnaJ4PmpibQO1GyUBiFzp9MttHIu5B2EiO5w7v+DgZMhWAOoRRqKhthtJNJo5ReV8uC763g9KY0ze0Xy9FVDCGru73VYTVtZGWz73Gll2fKx06rSaxwMnQ5dR4OfNsAUaWiUvIjUk20Zudw6bwWb03P41Vk9uGNMnDZY9NKRg87GiMtegMMpzmJyp98FQ6ZCaLTX0YnIcSh5EakH/129h/veXEPzAD9emjqU0Vp8zhvWwu7lTitL+TTnLqfCmAeh9y+0ZL+Ij1DyIlKHCktKeeS9jcxdksKQLuE8eeUgzSjyQlEerFvgJC17VzvTnAdfDYnToX2819GJyAlS8iJSR1IP5XHLvBWs3Z3F9aO6cs/5vWnmr/ET9cZa2LMSVr8Ka+ZDQRZE9oELHof+l2uas4gPU/IiUgc+Wb+Pu95YDcDMq4dwbt8OHkfUhOTsc5KVVa9AxibwbwF9xsPQ6yBmpKY5izQCSl5EalFxaRl/+2gTz329g4SoUJ6+ajCdI4K9DqvxKy6Aze87Ccu2L8CWQfQwZzG5vhdDULjXEYpILVLyIlJL9mblc9srK1meksk1I7tw/wV9aBGgadB1pnyPoVXzYN3bUJgFraPhtDthwJXQtqfXEYpIHVHyIlILvtqSwR2vraSopIwnrxzELwZ08jqkxutwKqx5DVa9Coe2QUAQxF/oLCQXe7rWZRFpApS8iJyE0jLLPz/bwlNfJtOrfSv+c9VgumtjxdpXdAQ2vue0suxYDFhnivOo3zg7OWvwrUiTouRF5Gfan1PAr19dxXfbD3JZYjQPX9hPq+XWppIiZ+XbtQtg84dQfATCusDo+5zZQhFdvY5QRDyi5EXkZ1i/J4tpLy0ju6CYxy7tz6TEzl6H1DiUlcLOr52EZeO7zvTmoAjof5mzi3PMSHULiYiSF5ETtWT7Qa6fnUSrwADeufVUendo7XVIvq184O3aBbD+bTiy31lErvd46HcJdD8T/LVxpYj8SMmLyAn4dEM6t76ygpiIYOZOH0bHUK2W+7NYC+nrnIRl3VuQtctZjyXuPCdhiTsPmunaikjVlLyI1NAbSanc99ZaEqJCeWnqUMJbah+cE3YgGda96SzVf2ALGH/ofhacdb+zk3OgWrFE5KcpeRGpgZmLt/HnDzYxqmdbnv3lEFq20K9OjWXuhPXvwPq3nH2FMM5MoRE3Q58J0LKN1xGKiI/RJ7DIcVhr+etHm3n2q21c0L8j/7hsgBaeq4nMFNjwjjOGZc9KpyxqCJz3Z2fF29ZaB0dEfj4lLyLVKCkt4/631zE/KZVfjojh4Qv74e+nfXGqdTj1x4Rl93KnrNNgOOdPzlos4V28jU9EGg2fS16MMecD/wL8geettY96HJI0QgXFpfz6tZV8vD6d28/uyZ1jemK0od+xstJgw0InYUlb5pR1HAhjHoa+F0F4rKfhiUjj5FPJizHGH/gPcA6QBiwzxrxrrd3gbWTSmOQUFHPDnOV8t/0gD/4inmmnajG0o2TtrpCwLHXKOvSHsx90EpaIbt7GJyKNnk8lL8AwINlaux3AGPMaMAFQ8iK14kBuIVNfWsqmvTn88/KBXDQoyuuQGoYDW51Vbje9D6lLnLL2CXDW/zljWNp09zY+EWlSfC15iQJSKzxOA4Z7FIs0MmmZeVz9wlL2ZuXz3DWJnNm7ndcheae0xGlV2fyBk7QcTHbK2yfAmX9wWli0a7OIeMTXkpeqBh3YoyoYcwNwA0BMTEx9xCSNwJb0HK5+4Xvyi0p5efpwEmMjvA6p/hXmQPLnTrKy9RPIPwR+zaDrKBh+k7NwXJh+p0TEe76WvKQBFTeRiQb2VKxgrZ0JzARITEw8KrERqcrylEyunbWMFgF+vH7TyKa13P/hVNjykZOw7PwaSosgKBx6nge9xjoLyGnhOBFpYHwteVkG9DTGdAV2A1cAk70NSXzZV1syuGnuctq1bsHL04fTOSLY65DqlrWwd5WTrGz+APatdcojusPwGyFuLHQeDv6+9tEgIk2JT31CWWtLjDG3AR/jTJV+0Vq73uOwxAcdzitixuLtPP/1dnq2a8Xsa4cR2aqF12HVjZJC2PE1bH7fSVpy9oLxg84j4Jw/Osvya/yKiPgQn0peAKy1HwAfeB2H+KYjhSW89L8dzFi8ndzCEi4c0Ik/XdSP1oGNbNfi/EzY+qkzOyj5cyjKgWYtocdZTrLS8zwtyy8iPsvnkheRn6OguJR53+/i6S+TOXikiDF92vPbc+Po07ERjefITHFnB30AO/8HthRC2kPCJU7C0vUMaBbodZQiIidNyYs0asWlZSxYnsa/P9/K3qwCTu3RhrvO7cWgmHCvQzt55eNXNrkJS/o6pzyyN5z6a+h9gbM8v5+ft3GKiNQyJS/SKJWVWf67Zg9PfLqFnQfzGBQTxuOTBnBKj7Zeh3ZyCnMh5VvY+rEzfiV7tzN+JWYknPuI08KiBeNEpJFT8iKNirWWzzbu5/FPNrNpXw69O7TihSmJnNW7nW/uTVRaDGlJsOMr2L7I2T+orASaBTvTmM/6g8aviEiTo+RFGo3/JR/gsY83syr1MF3btuTfVw5ifEJH/HxpJ2hrYf8GJ1HZvshpZSnKBQx0GgSn/Aq6jXZmCmn8iog0UUpexOet2JXJ3z/ezLfbDtIpNJC/XpLAJYOjCfD3kbEeh3e5ycpXTgvLkQynvE0PGHCFk6zEnuYsHiciIkpexLcUFJeyfk82a9MOs2Z3FmvSskjen0vbkOY8+It4Jg+PoUWAv9dhHl9WGqR+76y9suMrOLTdKQ9p73QFdT0Dup0BodHexiki0kApeZEGq6ikjM37cliddpi1aVms2Z3FlvQcSsucXR8iW7VgQHQoVwztzJXDYmjZogH+OJcUwt41ziaHqe4tx93Ronkrp0Vl2I1OshLZG3xxXI6ISD1rgJ/20hSVlJaxdX+um6QcZk1aFpv25lBUWgZAeHAz+keHMaZPOxKiQhnQOYz2rRvgmI/svT8mKmnLYM8qKC10zoXGQJeRzvL70UOhQwL4N7LF8URE6oGSF6k3JaVl7M0qIPVQHimH8thVfjuYx9b9ORQUO4lKq8AAEqJCufa0rvSPDiUhKpTo8KCGN1uotNjZGyh1qZuwLIOsXc45/xbQaSAMu95JVjoPg1YdvI1XRKSRUPIitSq3sIRdB/PYdejID8lJysE8Ug/lkZaZT0nZjxt9N/M3RIcHExMRzORhXRjQOZT+0WF0iQhueDOEio5A+nrYt8bpBtq31pkVVFLgnG8d5bSmjLjJSVY6JEBAI90rSUTEY0pe5IRlFxSz88ARdhw4ws4Deew86BzvOpTHoSNFR9UNC25GTEQwfaNCGZfQkZiIYGLaOAlLx9Ag/BtakgKQmwH7VjsJSnmicjAZcBOvwDDo2B+GXgdRQ5xWFQ2uFRGpN0pepEp5RSXHJCc7Dxxh58EjHMg9OkHpFBpIlzYtOa9ve2IiWhITEUyXNsF0jggmNKgBj+koLXamKe9b67SolCcruft+rBMa4yQqCZdCh/5Oi0potAbWioh4SMlLE2at5UBuEVvSc9i8L4et+3PYnuEkKOnZhUfVbdeqBbFtW3J27/bEtm1J17bBxLZtSZeIlgQ1b6BTk4vznWnJh1PgcCpkpR59n7MHrDPOBuPvzPbpfqaToHToDx36aW0VEZEGSMlLE5GVX8zW9Bw2p+ewZZ97n557VDdPeHAzukWGcFqPSLq2DaZr2xBi2wYT26Zlw5uGXHQEjhyAvAOQk+4mJLuOTlDKF3srZ/whNMppTek6CkI7Q3gXaN8XIvtoxVoRER/RwP4iyckqKC4leX8um/flsCU9h03u/d6sgh/qtGzuT1yHVpwb35649q3o1aEVce1b0TakuTczeqyFgizIO/hjQvLD/UHn/odz7n1J/rGvExDoJCRhnZ3Wk7DOTqIS1hnCYqBVR/BroK1EIiJSY0pefExpmWVvVj5pmfmkHsojNTOftMw80g7lk5qZx77sAqw7rrR5gB89IkMY0a2Nm6SEENe+FVFhdTjtuLQE8jOdJCP/EOQdquLePV9elp/pbDZYlWYtnU0Hg9tCSDtoFw/BEdCyrVPWsi20bOckKC0jNRZFRKQJUPLSwFhr2Z9TSFpmHqmH8n+YYpyamUdqZh57DxccNd3YGOjYOpDo8GBGdm9DTETwD60pXSKCT25/n7LSColG5duhKo4PQWFW9a/n38JJPIIinPvIXhDcxjkOdhOUlm0qHLeFZkE/P34REWmUlLx4yFrLrkN5rN2dxdq0LNbuzmLd7iyyC45uhYhs1YLo8CAGdQ7nwgFBRIcH0zk8mOjwIDqFBdE8oAYJirXOOJEfumAqd8ccqJSQHIT8w/wwPbiyZsFHJx7hsUc/Dgo/OlEJbuM8Ry0jIiJykjxJXowxk4CHgD7AMGttUoVzvwOmA6XA7dbaj93y84F/Af7A89baR+s77pNxVKLiJisVE5Xm/n707tiK8QM60btDKzpHBNM53ElUAptVMU6jpMjpcjmYUk2LyIEfx4iU30oKjn0dAL9mbuLRxmn56JDw4+MfbhE/HgdFQPPgOrxaIiIi1fOq5WUdMBGYUbHQGBMPXAH0BToBnxlj4tzT/wHOAdKAZcaYd621G+ov5JqrnKisc5OVqhKVhKhQEjq1Ji7M0rzgAOTuhyPbIOcQpJcnI5VbRDKhMLv6AJq3+nGcSKuObjIS8WNXTOUumhat1SIiIiI+w5PkxVq7Eahq0OgE4DVrbSGwwxiTDAxzzyVba7e7z3vNretZ8mKtJSOnkJ0H835YvC3lYB47Dhwh5eARjhSVEMoROvnnMKhNEeNjiogLyadz81za2MP452VAxn7YkQFH9kNpUdVv1DykQldMBLTpUaElpFJrSHm5lqUXEZFGrKGNeYkCllR4nOaWAaRWKh9e1QsYY24AbgCIiYk5qWCstezPLiAl/SD79u3hQEY6WQfTOZJ1gKKcgwSXZhNmcgkjlwF+Rzg3IJ82/kcIbZFDS//D+Ft37Eq2ewNnrZGWkRAS6cySieztPm7nPA5p9+NMGiUiIiIix6iz5MUY8xlQ1Ta691trF1b3tCrKLFDViNQqR5Jaa2cCMwESExOrGW16fBn7UsmdeYGTnJBLe1N8bCU/51bm14yyoAj8gyMwwREQFAtBYT9O7W3Z7sdEJaS9M5DV7yRmAImIiDRxdZa8WGvH/IynpQGdKzyOBva4x9WV17rQ0HBSAztzODicAyFtCA6LJDSiHaFt2uFfPoMmKByCwvFrFoSfxouIiIjUm4bWbfQu8Iox5h84A3Z7AktxWmR6GmO6ArtxBvVOrqsgmgeFMPie9+vq5UVEROQkeDVV+mLgSSASeN8Ys8pae561dr0x5nWcgbglwK3W2lL3ObcBH+NMlX7RWrvei9hFRETEW8banzUsxCckJibapKSkn64oIiI/MMYst9Ymeh2HSHU0clRERER8ipIXERER8SlKXkRERMSnKHkRERERn6LkRURERHyKkhcRERHxKY16qrQxJgfY7HUcVWgLHPA6iGo01NgU14lRXCdGcR2ti7U20oP3FamRhrbCbm3b3BDXKjDGJDXEuKDhxqa4ToziOjGKS8S3qNtIREREfIqSFxEREfEpjT15mel1ANVoqHFBw41NcZ0YxXViFJeID2nUA3ZFRESk8WnsLS8iIiLSyCh5EREREZ/SKJIXY8z5xpjNxphkY8x9VZxvYYyZ757/3hgTWw8xdTbGfGmM2WiMWW+M+XUVdUYbY7KMMavc2wN1HZf7vjuNMWvd90yq4rwxxvzbvV5rjDGD6yGmXhWuwypjTLYx5o5KdertehljXjTG7DfGrKtQFmGM+dQYs9W9D6/muVPcOluNMVPqIa7HjDGb3O/V28aYsGqee9zvex3E9ZAxZneF79e4ap573N/fOohrfoWYdhpjVlXz3Lq8XlV+PjSEnzERn2Ct9ekb4A9sA7oBzYHVQHylOrcAz7rHVwDz6yGujsBg97gVsKWKuEYD73lwzXYCbY9zfhzwIWCAEcD3HnxP9+EslOXJ9QJOBwYD6yqU/Q24zz2+D/hrFc+LALa79+HucXgdx3UuEOAe/7WquGryfa+DuB4C7qrB9/q4v7+1HVel848DD3hwvar8fGgIP2O66eYLt8bQ8jIMSLbWbrfWFgGvARMq1ZkAzHaPFwBnG2NMXQZlrd1rrV3hHucAG4GounzPWjQBmGMdS4AwY0zHenz/s4Ft1tqUenzPo1hrFwOHKhVX/DmaDVxUxVPPAz611h6y1mYCnwLn12Vc1tpPrLUl7sMlQHRtvd/JxFVDNfn9rZO43M+Ay4BXa+v9auo4nw+e/4yJ+ILGkLxEAakVHqdxbJLwQx33Qz4LaFMv0QFuN9Ug4PsqTo80xqw2xnxojOlbTyFZ4BNjzHJjzA1VnK/JNa1LV1D9HxQvrle59tbaveD88QHaVVHH62t3LU6rWVV+6vteF25zu7NerKYLxMvrNQpIt9ZureZ8vVyvSp8PvvAzJuK5xpC8VNWCUnn+d03q1AljTAjwJnCHtTa70ukVOF0jA4AngXfqIybgVGvtYGAscKsx5vRK5728Xs2BC4E3qjjt1fU6EV5eu/uBEmBeNVV+6vte254BugMDgb04XTSVeXa9gCs5fqtLnV+vn/h8qPZpVZRpzQtpUhpD8pIGdK7wOBrYU10dY0wAEMrPa+I+IcaYZjgfTPOstW9VPm+tzbbW5rrHHwDNjDFt6zoua+0e934/8DZO031FNbmmdWUssMJam175hFfXq4L08u4z935/FXU8uXbuoM3xwFXW2ir/kNXg+16rrLXp1tpSa20Z8Fw17+fV9QoAJgLzq6tT19erms+HBvszJtKQNIbkZRnQ0xjT1f2v/Qrg3Up13gXKR+RfCnxR3Qd8bXH7018ANlpr/1FNnQ7lY2+MMcNwvh8H6ziulsaYVuXHOIM911Wq9i5wjXGMALLKm7LrQbX/DXtxvSqp+HM0BVhYRZ2PgXONMeFuN8m5blmdMcacD9wLXGitzaumTk2+77UdV8VxUhdX8341+f2tC2OATdbatKpO1vX1Os7nQ4P8GRNpcLweMVwbN5zZMVtwZi3c75b9EefDHCAQpxsiGVgKdKuHmE7DacpdA6xyb+OAm4Cb3Dq3AetxZlgsAU6ph7i6ue+32n3v8utVMS4D/Me9nmuBxHr6PgbjJCOhFco8uV44CdReoBjnP93pOOOkPge2uvcRbt1E4PkKz73W/VlLBqbVQ1zJOGMgyn/OymfWdQI+ON73vY7jmuv+/KzB+aPcsXJc7uNjfn/rMi63fFb5z1WFuvV5var7fPD8Z0w33Xzhpu0BRERExKc0hm4jERERaUKUvIiIiIhPUfIiIiIiPkXJi4iIiPgUJS8iIiLiU5S8iFRgjCk1R+9ufdxdjo0xNxljrqmF991ZWwvuuWuUfOoef+MuyCYi0mjoQ03kaPnW2oE1rWytfbYug/mZRgJL3AXMjtgfN20UEWkU1PIiUgNuy8hfjTFL3VsPt/whY8xd7vHtxpgN7kaEr7llEcaYd9yyJcaY/m55G2PMJ8aYlcaYGVTYr8YY80v3PVYZY2YYY/zd2yxjzDpjzFpjzJ1VxNjdGLMKeBmYDCwHBrivU9UGfyIiPknJi8jRgip1G11e4Vy2tXYY8BTwzyqeex8wyFrbH2dlYICHgZVu2e+BOW75g8A31tpBOKvPxgAYY/oAl+NsCjgQKAWuwtncMMpa289amwC8VPnNrbXb3Ocsx9mHZw7OirIDrbM/j4hIo6BuI5GjHa/b6NUK909UcX4NMM8Y8w4/7nh9GnAJgLX2C7fFJRQ4HWdjQKy17xtjMt36ZwNDgGXuNk5BOJvz/RfoZox5Engf+OQ4X0M7a+1BY0wCzoaIIiKNilpeRGrOVnNc7gKcPaGGAMvdgbKminq20n1FBpjttpYMtNb2stY+ZK3NBAYAi4BbgeePeaIxzxpj1uFsdLgKOB94v6ouJhERX6bkRaTmLq9w/13FE8YYP6CztfZL4B4gDAgBFuN0+2CMGQ0csNZmVyofC4S7L/U5cGn5GBV3zEwXdyaSn7X2TeD/gMGVg7PW3oTTTfUn4CLgfTcBqqqVSETEZ6nbSORoQW6rRbmPrLXl06VbGGO+x0n6r6z0PH/gZbdLyABPWGsPG2MeAl4yxqwB8oApbv2HgVeNMSuAr4BdANbaDcaYPwCfuAlRMU5LS777OuX/cPyumvjPwBnrMsp9XRGRRke7SovUgDFmJ5BorT3gdSwiIk2duo1ERETEp6jlRURERHyKWl5ERETEpyh5EREREZ+i5EVERER8ipIXERER8SlKXkRERMSn/H86NxJMW3jqTQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml2-kernel",
   "language": "python",
   "name": "ml2-kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
